package cn.haut.online_read.crawler.sanwenzx;

import cn.haut.online_read.utils.toMapList;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.*;

/**
 * @author xing
 * @date 2021-05-21 19:54
 */
public class CategoryCrawler implements PageProcessor {

    private Site site = Site.me().setSleepTime(1);

    @Override
    public void process(Page page) {
        List<String> categoryNameList = page.getHtml().xpath("//div[@class='nav mt1']/div/ul/li/a/text()").all();
        List<String> categoryUrlList = page.getHtml().xpath("//div[@class='nav mt1']/div/ul/li/a/@href").all();

        // 第一个是主页，就不要了
        categoryNameList.remove(0);
        categoryUrlList.remove(0);

        for (int i = 0; i < categoryNameList.size(); i++) {
            // 没必要有，删了
            if("爱情散文".equals(categoryNameList.get(i))) {
                categoryNameList.remove(i);
                categoryUrlList.remove(i);
            }
            if("散文精选".equals(categoryNameList.get(i))) {
                categoryNameList.remove(i);
                categoryUrlList.remove(i);
            }
        }

        List<Map<String, Object>> categoryList = toMapList.get(
                Arrays.asList("name", "url"),
                categoryNameList, categoryUrlList
        );

        page.putField("categories", categoryList);

        List<String> majorNameList = page.getHtml().xpath("//div[@class='listbox']/dl/dt/strong/a/text()").all();
        List<String> majorUrlList = page.getHtml().xpath("//div[@class='listbox']/dl/dt/strong/a/@href").all();

        // 存储每个 小类 里 所有 文章信息
        List<List<Map<String, Object>>> articlesList = new ArrayList<>();
        for(int i = 1; i < majorNameList.size() + 1; i++) {
            // 爬取信息
            List<String> articleDateList = page.getHtml().xpath("//div[@class='listbox']/dl[" + i + "]/dd/ul/li/span/text()").all();
            List<String> articleNameList = page.getHtml().xpath("//div[@class='listbox']/dl[" + i + "]/dd/ul/li/a/text()").all();
            List<String> articleUrlList = page.getHtml().xpath("//div[@class='listbox']/dl[" + i + "]/dd/ul/li/a/@href").all();
            // 分开放入 map
            List<Map<String, Object>> articleMapList = toMapList.get(
                    Arrays.asList("date", "name", "url"),
                    articleDateList, articleNameList, articleUrlList
            );
            // 放入总的 list
            articlesList.add(articleMapList);
        }

        // 存储 所有 小类 及其下边的文章信息 的 List
        List<Map<String, Object>> majorList = toMapList.get(
                Arrays.asList("name", "url", "articles"),
                majorNameList, majorUrlList, articlesList
        );

        // 放入major
        page.putField("major", majorList);

    }

    @Override
    public Site getSite() {
        return site;
    }
}
